{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "u0Bp-Metx3XM"
   },
   "source": [
    "# Restricted Boltzmann Machine as a Generative Model\n",
    "The following code is partly referenced from\n",
    "https://github.com/odie2630463/Restricted-Boltzmann-Machines-in-pytorch\n",
    "## Setup PyTorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "AVQfrTWR-43O"
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    import torch\n",
    "except:\n",
    "    !pip install torch torchvision\n",
    "    import torch\n",
    "import numpy as np\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "99w9A3nOfcVI"
   },
   "source": [
    "### make sure we are using GPU (Nvidia K80)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 52
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1247,
     "status": "ok",
     "timestamp": 1525697323635,
     "user": {
      "displayName": "刘金国",
      "photoUrl": "//lh3.googleusercontent.com/-lDAT81T3HSE/AAAAAAAAAAI/AAAAAAAAAgw/eH3JEob7M1Y/s50-c-k-no/photo.jpg",
      "userId": "116824001998056121289"
     },
     "user_tz": -480
    },
    "id": "L6uY7QZV_QHh",
    "outputId": "ae5d22a4-2cfb-490c-bebf-b8c6835b2711"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using CUDA device 0!\n",
      "tensor([ 1.,  2.,  3.], device='cuda:0')\n"
     ]
    }
   ],
   "source": [
    "has_cuda = torch.cuda.is_available()\n",
    "if not has_cuda:\n",
    "    print('Not Using CUDA!')\n",
    "else:\n",
    "    print('Using CUDA device %d!'%torch.cuda.current_device())\n",
    "    ts = torch.Tensor([1,2,3])\n",
    "    ts = ts.cuda()\n",
    "    # you will see the location information of a tensor by printing it out.\n",
    "    print(ts)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "sn9DboRcfkuf"
   },
   "source": [
    "## Code an RBM\n",
    "#### Derivative of Negative log-likelihood\n",
    "$\\frac{\\partial{\\mathcal{L}}}{\\partial \\theta}=\\langle \\frac{\\partial E_\\theta(x)}{\\partial \\theta}\\rangle_{x\\sim\\mathcal{D}}-\\langle \\frac{\\partial E_\\theta(x)}{\\partial \\theta}\\rangle_{x\\sim p_{\\theta}(x)}$\n",
    "\n",
    "#### Free energy\n",
    "$E_\\theta(v) = -\\log(p_\\theta(x))$\n",
    "\n",
    "#### $k$-th order Contractive divergence (CD-$k$)\n",
    "Foward Gibbs sampling $x_0\\rightarrow h_0$ + backward Gibbs sampling $h_0\\rightarrow x_1$ + $\\ldots$ + backward Gibbs sampling $h_{k-1}\\rightarrow x_k$.\n",
    "\n",
    "For $k=\\infty$, we will get exact $p_{\\rm \\theta}(x)$."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "JcDupOzQ_ec6"
   },
   "outputs": [],
   "source": [
    "class RBM(nn.Module):\n",
    "    '''\n",
    "    Restricted Boltzmann Machine\n",
    "\n",
    "    Args:\n",
    "        num_visible (int): number of visible nodes.\n",
    "        num_hidden (int): number of hidden nodes.\n",
    "\n",
    "    Attributes:\n",
    "        W (2darray): weights.\n",
    "        v_bias (1darray): bias for visible layer.\n",
    "        h_bias (1darray): bias for hidden layer.\n",
    "    '''\n",
    "\n",
    "    def __init__(self, num_visible, num_hidden):\n",
    "        super(RBM, self).__init__()\n",
    "        self.W = nn.Parameter(torch.randn(num_hidden, num_visible) * 1e-2)\n",
    "        self.v_bias = nn.Parameter(torch.zeros(num_visible))\n",
    "        self.h_bias = nn.Parameter(torch.randn(num_hidden) * 1e-2)\n",
    "        self.num_visible = num_visible\n",
    "        self.num_hidden = num_hidden\n",
    "\n",
    "    def _v_to_h(self, v):\n",
    "        '''\n",
    "        forward pass p(h|v) from visible to hidden, v is visible input.\n",
    "        '''\n",
    "        p_h = F.sigmoid(F.linear(v, self.W, self.h_bias))\n",
    "        return p_h\n",
    "\n",
    "    def _h_to_v(self, h):\n",
    "        '''\n",
    "        backward pass p(v|h) from hidden to visible, h is hidden input.\n",
    "        '''\n",
    "        p_v = F.sigmoid(F.linear(h, self.W.t(), self.v_bias))\n",
    "        return p_v\n",
    "\n",
    "    def contrastive_divergence(self, v, k):\n",
    "        '''\n",
    "        Args:\n",
    "            v (ndarray): visible input.\n",
    "            k (in): CD-k, means k times v->h & h->v sweep in a single contrastive divergence run.\n",
    "\n",
    "        Returns:\n",
    "            ndarray: visible obtained through CD sampling.\n",
    "        '''\n",
    "        prob_h = self._v_to_h(v)\n",
    "        h = sample_from_prob(prob_h)\n",
    "        for _ in range(k):\n",
    "            prob_v = self._h_to_v(h)\n",
    "            v = sample_from_prob(prob_v)\n",
    "            prob_h = self._v_to_h(v)\n",
    "            h = sample_from_prob(prob_h)\n",
    "        return v\n",
    "\n",
    "    def energy(self, v):\n",
    "        '''\n",
    "        free energy E(x) = -log(\\sum_h exp(x, h)) = -log(p(x)*Z).\n",
    "        It can be used to obtain negative log-likelihood L = <E(x)>_{data} - <E(x)>_{model}.\n",
    "\n",
    "        Args:\n",
    "            v (1darray,2darray): visible input with size ([batch_size, ]data_size).\n",
    "\n",
    "        Return:\n",
    "            float: the free energy loss.\n",
    "        '''\n",
    "        vbias_term = v.mv(self.v_bias)\n",
    "        wx_b = F.linear(v, self.W, self.h_bias)\n",
    "        hidden_term = wx_b.exp().add(1).log().sum(dim=-1)\n",
    "        return (-hidden_term - vbias_term).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "LEVdcofVU7Wx"
   },
   "outputs": [],
   "source": [
    "def sample_from_prob(prob_list):\n",
    "    '''\n",
    "    from probability to 0-1 sample.\n",
    "\n",
    "    Args:\n",
    "        prob_list (1darray): probability of being 1.\n",
    "\n",
    "    Returns:\n",
    "        1darray: 0-1 array.\n",
    "    '''\n",
    "    rand = torch.rand(prob_list.size())\n",
    "    if prob_list.is_cuda:\n",
    "        rand = rand.cuda()\n",
    "    return (1+torch.sign(prob_list - rand))/2.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "6KHy9kQCfqvf"
   },
   "source": [
    "## Load Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "JugGWtAqVLfJ"
   },
   "outputs": [],
   "source": [
    "def mnist01_loader(is_train, use_cuda, batch_size):\n",
    "    '''\n",
    "    yield image and label from mnist dataset.\n",
    "\n",
    "    Args:\n",
    "        is_train (bool): yield traning set if True, else test set.\n",
    "        use_cuda (bool): return data on GPU in True.\n",
    "        batch_size (int): size of a batch.\n",
    "\n",
    "    Returns:\n",
    "        func: an iterator function.\n",
    "    '''\n",
    "    from torchvision import datasets, transforms\n",
    "    import torch.utils.data\n",
    "    test_loader = torch.utils.data.DataLoader(\n",
    "        datasets.MNIST('data_torch', train=is_train,\n",
    "                    download=True,\n",
    "                       transform=transforms.Compose([\n",
    "                           transforms.ToTensor()\n",
    "                       ])), batch_size=batch_size)\n",
    "\n",
    "    def iterator():\n",
    "        for data, label in test_loader:\n",
    "            # transform to binary mnist image\n",
    "            data = data.view(-1, 784)\n",
    "            data = data.bernoulli()\n",
    "            if use_cuda:\n",
    "                # copy data to gpu memory\n",
    "                data = data.cuda()\n",
    "                label = label.cuda()\n",
    "            yield data, label\n",
    "    return iterator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 301
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1062,
     "status": "ok",
     "timestamp": 1525697331131,
     "user": {
      "displayName": "刘金国",
      "photoUrl": "//lh3.googleusercontent.com/-lDAT81T3HSE/AAAAAAAAAAI/AAAAAAAAAgw/eH3JEob7M1Y/s50-c-k-no/photo.jpg",
      "userId": "116824001998056121289"
     },
     "user_tz": -480
    },
    "id": "1R8AUsRIVOia",
    "outputId": "11b10497-bd89-4583-b332-937e99e3416d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz\n",
      "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz\n",
      "Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz\n",
      "Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz\n",
      "Processing...\n",
      "Done!\n",
      "torch.Size([64, 784]) torch.Size([64])\n",
      "A hand written digit with label \"5\"\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAABA1JREFUeJzt3dFR20AUQNGYoYpUkSaYVECVVJCh\niVSRMhAVIHvYrFfWPefb2Ptz5308rbhs2/YD6HlafQBgDfFDlPghSvwQJX6IEj9EiR+ixA9R4oeo\n53v+2MvTq8cJYbL3j7fLLZ8z+SFK/BAlfogSP0SJH6LED1HihyjxQ5T4IUr8ECV+iBI/RIkfosQP\nUeKHKPFDlPghSvwQJX6IEj9EiR+ixA9R4oco8UOU+CFK/BAlfogSP0SJH6LED1HihyjxQ5T4IUr8\nECV+iBI/RIkfosQPUc+rD8C5/fn3d/URpvj989fqIwwz+SFK/BAlfogSP0SJH6LED1Hihyh7/ju4\ntuse3Rnvff+17z7yHn7l2Ud/+xGeAzD5IUr8ECV+iBI/RIkfosQPUZdt2+72Yy9Pr/f7sQM58jrt\nzB5h3TbD+8fb5ZbPmfwQJX6IEj9EiR+ixA9R4oco8UOUK70MmXnttrqnvxeTH6LED1HihyjxQ5T4\nIUr8ECV+iLLnv4OZr+aebfTse3/vPQdrmfwQJX6IEj9EiR+ixA9R4oco8UOUPf8DGNm1H/lfSbuv\nv5bJD1HihyjxQ5T4IUr8ECV+iBI/RNnzn8DIvfgjPwfAXCY/RIkfosQPUeKHKPFDlPghyqrvBLwe\nm+8w+SFK/BAlfogSP0SJH6LED1Hihyh7fnaNXvnd+3vXhdcy+SFK/BAlfogSP0SJH6LED1Hihyh7\n/pMb2cPfwvsCHpfJD1HihyjxQ5T4IUr8ECV+iBI/RNnzx43eqZ/578Gv8T6AMSY/RIkfosQPUeKH\nKPFDlPghSvwQZc/PkNnvCxj5bs8B7DP5IUr8ECV+iBI/RIkfosQPUVZ9TLW3bvPa77VMfogSP0SJ\nH6LED1HihyjxQ5T4Icqen6lGdvkrrwsXmPwQJX6IEj9EiR+ixA9R4oco8UOUPT+7Vu7SvZp7LpMf\nosQPUeKHKPFDlPghSvwQJX6Isuc/OXfe+YrJD1HihyjxQ5T4IUr8ECV+iLLqO4Ejr/Ncuz0ukx+i\nxA9R4oco8UOU+CFK/BAlfoiy5/8PjrxnH2VPf14mP0SJH6LED1HihyjxQ5T4IUr8EJXZ8595F7/H\nnp6vmPwQJX6IEj9EiR+ixA9R4oco8UNUZs9/bd+98jmA0bPZ5fMdJj9EiR+ixA9R4oco8UOU+CFK\n/BCV2fNfc+Rd+ZHPxuMy+SFK/BAlfogSP0SJH6LED1HihyjxQ5T4IUr8ECV+iBI/RIkfosQPUeKH\nKPFDlPghSvwQJX6IEj9EiR+ixA9Rl23bVp8BWMDkhyjxQ5T4IUr8ECV+iBI/RIkfosQPUeKHKPFD\nlPghSvwQJX6IEj9EiR+ixA9R4oco8UOU+CFK/BAlfogSP0SJH6I+AaVgdzlFtJnkAAAAAElFTkSu\nQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "loader = mnist01_loader(is_train=True, use_cuda=False, batch_size=64)\n",
    "# let's check the data and labels\n",
    "for data, label in loader():\n",
    "    print(data.shape, label.shape)\n",
    "    print('A hand written digit with label \"%d\"'%label[0])\n",
    "    plt.imshow(data[0].data.numpy().reshape([28, 28]))\n",
    "    plt.axis('off')\n",
    "    plt.show()\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "U0gdtiqt34_k"
   },
   "source": [
    "## Use RBM as a Generative Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 407
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 262070,
     "status": "ok",
     "timestamp": 1525697593682,
     "user": {
      "displayName": "刘金国",
      "photoUrl": "//lh3.googleusercontent.com/-lDAT81T3HSE/AAAAAAAAAAI/AAAAAAAAAgw/eH3JEob7M1Y/s50-c-k-no/photo.jpg",
      "userId": "116824001998056121289"
     },
     "user_tz": -480
    },
    "id": "gXQOWrPYVUrS",
    "outputId": "8de2ec7e-d9d3-4963-f7ba-ec3912c082ce"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 0, Mean \"Loss\" = -8.4453, Elapse = 12.4634\n",
      "epoch 1, Mean \"Loss\" = -6.6504, Elapse = 12.4389\n",
      "epoch 2, Mean \"Loss\" = -4.5493, Elapse = 13.7533\n",
      "epoch 3, Mean \"Loss\" = -3.1713, Elapse = 12.2471\n",
      "epoch 4, Mean \"Loss\" = -2.2299, Elapse = 11.2361\n",
      "epoch 5, Mean \"Loss\" = -1.5794, Elapse = 11.0825\n",
      "epoch 6, Mean \"Loss\" = -1.0708, Elapse = 12.7443\n",
      "epoch 7, Mean \"Loss\" = -0.7211, Elapse = 12.8113\n",
      "epoch 8, Mean \"Loss\" = -0.5161, Elapse = 18.6288\n",
      "epoch 9, Mean \"Loss\" = -0.2794, Elapse = 14.0293\n",
      "epoch 10, Mean \"Loss\" = -0.1166, Elapse = 15.0979\n",
      "epoch 11, Mean \"Loss\" = -0.0782, Elapse = 12.8121\n",
      "epoch 12, Mean \"Loss\" = 0.1309, Elapse = 10.7571\n",
      "epoch 13, Mean \"Loss\" = 0.2717, Elapse = 10.6880\n",
      "epoch 14, Mean \"Loss\" = 0.3338, Elapse = 10.8203\n",
      "epoch 15, Mean \"Loss\" = 0.3237, Elapse = 10.6227\n",
      "epoch 16, Mean \"Loss\" = 0.4811, Elapse = 10.7536\n",
      "epoch 17, Mean \"Loss\" = 0.5352, Elapse = 10.7733\n",
      "epoch 18, Mean \"Loss\" = 0.4947, Elapse = 10.6619\n",
      "epoch 19, Mean \"Loss\" = 0.5675, Elapse = 12.9142\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "use_cuda = True   # if you don't have cuda, this line must be set False!\n",
    "num_visible = 784\n",
    "num_hidden = 500\n",
    "\n",
    "# set seed for pytorch-cpu, pytorch-gpu and numpy\n",
    "seed = 10086\n",
    "torch.manual_seed(seed)\n",
    "if use_cuda: torch.cuda.manual_seed_all(seed)\n",
    "np.random.seed(seed)\n",
    "\n",
    "# define network and dataset, and transfer model data into GPU memory\n",
    "rbm = RBM(num_visible, num_hidden)\n",
    "if use_cuda: rbm = rbm.cuda()\n",
    "loader = mnist01_loader(True, use_cuda, batch_size=64)\n",
    "\n",
    "# with the stochastic gradient descent optimizer,\n",
    "# we optimize model parameters with learning rate 0.1.\n",
    "train_op = torch.optim.SGD(rbm.parameters(), 0.1)\n",
    "for epoch in range(20):\n",
    "    t0 = time.time()\n",
    "    loss_list = []\n",
    "    for data, label in loader():\n",
    "        # calculate the \"loss\", the last node in computation graph\n",
    "        v1 = rbm.contrastive_divergence(data, k=1)\n",
    "        loss = rbm.energy(data) - rbm.energy(v1)\n",
    "        loss_list.append(loss.item())\n",
    "\n",
    "        # get gradients using back propagation.\n",
    "        # zero_grad are needed before backward, otherwise gradients are accumulated.\n",
    "        train_op.zero_grad()\n",
    "        loss.backward()\n",
    "        # update parameters using gradients using gradients.\n",
    "        train_op.step()\n",
    "    t1 = time.time()\n",
    "    print('epoch %d, Mean \"Loss\" = %.4f, Elapse = %.4f'%(epoch, np.mean(loss_list), t1-t0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 270
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1379,
     "status": "ok",
     "timestamp": 1525697595114,
     "user": {
      "displayName": "刘金国",
      "photoUrl": "//lh3.googleusercontent.com/-lDAT81T3HSE/AAAAAAAAAAI/AAAAAAAAAgw/eH3JEob7M1Y/s50-c-k-no/photo.jpg",
      "userId": "116824001998056121289"
     },
     "user_tz": -480
    },
    "id": "XDweacKvVzhE",
    "outputId": "18dafe7f-1c67-4466-88c5-672af84a0ff3"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAO8AAAD8CAYAAACfMvOMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAEkRJREFUeJztneFy7SYShNHWvv8rn/2RaC8hgJDo\nHmipv6pbznFskIBmhgHGx+/3S8YYPf6z+gGMMc+weI0RxeI1RhSL1xhRLF5jRLF4jRHF4jVGFIvX\nGFEsXmNE+e/qB0gppeM4fMzLmIzf73dc/YwtrzGiWLzGiGLxGiOKxVtw3rL6/X7//2fMjli8Gblw\na99n18so15PPe7F40z8H+XEc1a9qMCeEVfUiyh71qFCeF3Py3F68ke5rLtjcCjMFzHivmueArGeV\ngJGM9OlxHFN9z26HLfZ5R7lqjKcNffV7bAEjy+61Efs9UuJ6KqiyR/p7htrkyWiX7S1vSuMzIMNC\nz86+kfTeHfkOvbIiPImIctB9zmiX7S1v3ohlg+YzWrlmNZy2iG5ftHCjvax8CYZGwvK2YAi3Frxi\nwVyHtp79qXcyGuTZnfwZ2bsKNWODZHvL+3beYB1T2ttdTumfE31PwLP0Jn50v0hb3pTwVvdEyerm\nrAoYsbfWWMGqMqaBqqe2S4Luc3nxolBw+XqMBtaeDM7W4ZXazyhwtlWk11Mu8RBIu82stali0Ouq\nLWb2rfOfH1n3qvUHehzlQq2ViWojecurNlBWwFgOtKzXG9vvKey2kbW8zGN4dzbxZ9xQtZNbKcVt\ntSi54XdAbh1Jipc9+O80bsSppTusHvRIyx7xLqvOByDGjZx4mRYl78haPUiLwgqWsJ47mqhJsTzH\nfn6PXedZ30xdcuJlwtgyiKK8VJF/jX4GJaKFm/M5y6tOlDWJqqusc9fydqsvpQ9bXsVZ/q0o94Xy\ns0uJd7fgkDEzzI5lqX1eC9eYP0iJ1xjzB4vXGFEsXmNEsXhfiFO+fgOpaPMbiMpG6eDeGKvOmSPq\nkxYvK5skm9Zz7/q8q2EmSWCcoY7qX1m3WdEtvMozNevuvs3q1tqDJTRWhkrmxX8Zy9u7NBCZphNR\nV+sMdVSidwQsr6flYrIna8ZtKHYfSlje3myGbKBypq+VPzuIRvJAPWHkuRjWpdYXSOHm9Yz83F0Y\n6WmikBBvSvzk51eibf3sk/JXkFt1hGueEj616VVyOAYMKxk1IWwv3tWD/mR3Nzal9mCvtSFaaIh+\nuuOKRwWZZsuqZZFEsf2aN2K9MzL7RgWSZl3n3qyPsgiR1xqjQNVX80xYsYytxRsdPR0Z1E+e5c4E\nNPPO7IRnrXdAbuUoeDg9as/f8lI+casodz0YbkjpbrYaNWpgzQZ8RmZ4RHCMEfXfZZnEBLWW31q8\nVy+IFnKvUdmR4JmfPylT4aDraG1rRaJumXNmx+7W4j0Z6TDmIFJZf93ZLpp5p9ZWHSqKbcbYXry5\nNWGv6U6Yx/FG652ldXpr1mXreSaocnvt8KZTZJ9Y87ZQOnXTonT9GUEm9oCPaKeU8MJljJ87ZX42\nAV3Jzh16p+ynHXquc6MvPSgc5dwNhAeUkqB4mWebV99eQW2zlPu97OUFso58ElKdFKLW78cOgYLj\nOJY/BPuywxthu+OrYg878Pv9Ll9YzvJG8LWB8hT26Tf3Qx/pgBWS2RsxX8XttQ6LN8MD0Shh8Roj\nisVrjCgWrzGiWLzGiGLxmm1hZaHY4WwDAu/zkvHhj3nO/WT1+8LorKSSlpeVE4hFeWNmxfNHpRJC\nl/eG/feyv1HZNqUsbzlAZmbjyAP8+UBcPfMz3w9d/u63oK4oBfupHFYtEA3QEhJL1LUsFwwxtcpk\nirb2PWVLiaZlYV9/JXDkcPrTA+xX92xrn5GJ4dAivrrEzrp9hV7Xs7yTO6mBIu4ov/4y/qoZnJ3q\n5RST0to9pWvhIstnUBtPtRiEQr/IWF601V1FKWA0rcG54n7ybPnRoCZpxO+PsL3l3YWng2n092Yj\n0OhtiKs62JNmvqRARudHYhqozJjsCWh7y3tSWo/yM7uhUFkXr8qftZJMEV95P4w+qIlqZiKttU8v\nqd4TWuJHt8/2lre2R5p/jiAigpqvr2ctTWu9zgQd4GE+90jboibpvA/Q43V78aZUD7H3/j8SZIOz\n14g9sUakqkGWdxU5Z9aD2oqseYbI9pIQb0r/tibI4EILhnDvlMkepKiyVSnbRu3knox4e0QcQECs\nF++WoRA9Twn7nCNlzbq0rYj8bNnRyASsakTtkzJOQUXVyWqjiCAVc3lUC36elpcRnS/rRCAtXvZZ\n3ZQ4+ZvLOpgwRLbCtVTYTSjLymE8v6x4FS1ur+x89ldy3ZjPqxYpjwykpvSSNS+SVQGLiC0SBLWr\nbeYvIs8dpCRsedmN85ZByXL7TZ3I9pEVLwsPTqOC3WZjRLF4jRHF4jVGFIvXGFEsXmNEsXg7rLh+\naP7C7X2N3FZR1Akk5m0cZSIydpgxpMTbSjWKPo+aX4qv/czug7TVJshsIBFpXxkT9dVkzKrPZ5tT\njHBGUoTOpkipsfukEAnrlk9+fpztVbHL95o3tW+B9M6qIjqGcRY2ws1Xy9h5Up4fv+pjVH0sZCxv\nxPW/GmXn3nWDRq+HId5vlXCjrzkiUXzmEwnxRl+TG6lr5JlGfobhvrEzRfTyZKFdXRbR1/cYbSLj\nNo/k20WU2SvvrshGnw1hua7Eudvkdwe2dcwnHUbZ+Vck24v3nLGiBt+KgBI6i0MvEqwIM9VOSjH7\n+YxytxdvSv2sfqhGqWWlHH0GRN1nHU9oiXXVXjWyT5Dl1crf3b3vsf2atyemvHMRwZ7RiPJuaV9G\nA2AKAbEV9UVMEoyytxdvC9Vtlqh6V04wu2e+jIb1DhJuM4sng4y9/mKgmjkyMsOm4iQhL17lKOqJ\n2sCprafZAoh2oRWQF+9sp67urIjz0ugAUlku68RVxN5rxGkxVrmya96c2YBVbW21WtSKKOzvRiVE\nj0BevKigxsoOVBo8ZWSW+exKsYCRutCnrI4d1lvHcax/iEWoXDM0sfx+v8sBIb/mVcbCNTPIu83K\nWLRmBlteY0TZYs1rjLmPLa8xoli8xohi8RojisVrjCgWrzGiWLzGiGLxGiOKxWuMKBavMaJYvMaI\nYvEaI4rFa4woFq8xoli8xohi8RojisVrjCgWrzGiWLzGiGLxGiPKFtkjv5y32ZgazttszIuxeI0R\nxeI1RpQt1ryrKf8A1Jv+khz6j1uZfbB4/6aXfF5x8DuZ/jXqE9v2bnPEIDw78DiOf/yL4PwDz8j3\nLMti/9X6Wt3o+lDvkLd3xB/vzr+i2d7yln9/l/F3TncA9T5sl3/0Lxvu3j8jz/d0nOV9wByrW4u3\n9+LoQVr+7pvcznLyQ5UZ6Z2kNPfs+fMyhRvJ1m5z7a/e90TFEhxbyCpWd7Rc5hIAVVarXIR7XjME\njDG0tXhTSv9af5Zr0vL/zaIaaS4tRdlG538jBpGyyzxqfWetfATbi/ekFkRCu7qnAHI3jTkQka5Z\n+dwMrp6XEaQ6mX2nkckL5Z7ndTLHz9Zr3rsgLW9UJBJd3irhMkF7VOxtwdpyj4GM5W0RJTYVRgbN\n3YE1suXBHKy99ekoo1Zw5j1q7cRsF3nxmn/CnMR6Za+YPJ/U2YoFoJ6nd1IPHbiSdpuRVrc3u7/B\nqs+sH0838E3bZyPxk6f9Xq6vy8+oMSVreVkDqRbdrtWNcK/QjAyImQFZ+90rESBgBw3zr8g6R8bS\nDNKWNyV8g+SzYs1a7Wx98pm+jJyf35+lJeCd2+WtSFreyHOp+Wf0AFWLaO9aJwLFpZGc5UWvG05y\n69GyvufPzcCaeMrnL/8fC0ZdjOOcu4EYB3LiZTKyptuZiIMaEXWucMOj2g35brLiVRJVTsTgWAG6\nXuXJZ6ROBFJrXtX11NtRD1ixjzHmINtJSrwp6Vrct+N+GQPZTlLiHb0RYswXkBKvMeYPFq8xoli8\nxohi8RojymvEq7BVMZKPC53e1FyDbqv8pBuzL2QPaeSUxwJ3jUbX9kORxwtbFyl2bY8Roo5KMi64\ntL7/yUMaJfmM1rsEjaxv5e8/qUtVuDVrxbBiI1lCZjgPgDC2OaUsb++aG2qw1urI65qdOXtlnAPz\nSfkrDvOXdaITuOXlsk9xqd1DTknI8l513OwRt3xG7wnoaR0j4kLcGe49H8pqleWwDs+0ymWIAjkx\nRE2eEpY38opbrx62KxqVxG3Ge2Ba+NH3Z6zjkeWh2vqK7S3vk8HyVATl+iQa9qSAvv9c1oFcP6pe\nzyyF2/p/CLYX78mdjrvbyb3IYP5vxq0dnRSQk0fp3s68wx2XeyZPVl6fIiM5zz51nzcqE8RVo84G\nk+787qy7xbwg31pOzNZ355l33gK7ijucXz+RSaMlKlT0N6+nF92csSh3Z1u2CGb2gVsWPFJMuwr3\nis9cCRyxhEgYUdO7a0FURLj1DjMTUjnRMfZbv8TsO28t3lExMToeaXXv1jk7eZyW/spjeVr2+XVl\ncA/d5ysmj9m221q8Kf17sKx8BlY5rZNiM2Xn7VVafuRSA8XdYBXiGVa5/LX6n7C9eHvWI2JrBVFH\nL8KLDvaUn8uvrHZDRoqvPIPdD2mM1AVZGu2w1jiOY/ghemJGwDyIETEgW8c6WaDbq7dcQVrKkWXR\n02BerbxakO8iKn1ZsUS0OQcZBS5RXPesKp81yUVZ3ZH9bqQnlH8P9g5qlpfF6vWPItHHRb/UL6+0\nvCxmIrBfRdVreAvbB6yiYFuRN4He4zXPsHj/xqIdZ+W2nfmDxZvhAWmUsHiNEcXiNUYUi9cYUSxe\nszVXR0q/jMVLBpka5mswjsLmmVGiKeuefQ5Z8aI6IKITy85aNYCY9bHu97J2AKJ2FpiTt9QJq5YL\nhTykjsjOUSu3BvNYYeS2F2tSUBfuSe2Y52fS4KTUPnv8ZOD0Gi4vb/a8c3nkspaJAj2Qoqx59O0l\nRXreA6KtthfvlYB6AmmR/0zrKljrfuxTEZdlMa8ctspm3L4qrcjOx0x7ExvrYkrPSLze8t65MIBw\nn1tH/2ayUNQGN2OwXNwPhS0v8j5Bv8+qQFLtM3MCQpS9fcDqqjORnc1eY7WeFRHsGbUqCCIENnLf\n9g53DEBEvAPB9uI92cUNu/scTxLQPYWdZWQEpGuO2lIZoZbza3e2F2/eoKsaFVVvnkzv/FrO9AgB\n91zaGWrPfRU/eFpH/vn8NzMGIsVZBj1ZY3d78ZZEWxd0o5cCPutADa68HHaiuZHv3yVfb47sCMzU\ng17bttb9iMmnhoR4a9HaFVYYnfSsLLe1TTXCqju2Ki5mjVJQs4JueVYsto82n/Qagbntwij3ynqp\nCIIRlc0FxbbyZXnIAzoRSFjeFmdDMwZ7be2FQkWcb6QmVOYRRibS2SOZsyRzr2+kzdGTBeskV2Ss\nIaovGGvgu+X+vpA9kingVfu+u7tt5dqcFRiLcGFzy8uKaSDKriEr3tWWEQFy4LRgH9DY/RRSdD1P\njus+RVa8LKKXEbtbWXOfsElHec1rzFsZWfNKR5uN+TIWrzGiWLzGiGLxGiOKxWuMKBavMaJYvA1W\n3h82ZgQf0riJ0q0TJlFngtmJ+tjn433CKsUchG/dUFKzwK1Bw0p6V97KmS2bOeBrd6bZR21Z7yPv\nNiNTvPSEi0pVU6ZFYaZJafF0MNUGO/O5WWXXUviw6kmJ9x4y4mXPxj03EDFIa3UwOpUppvIiRZm+\nhwErDVHrsxIy4o1ea5aXtWupeEboTQyrUtfM0LImau8RwUxaoxEk1rxRKVFKoda+3umEu2vM2Qmq\nt3ZD31lVF2vE5MNeVkhYXmYj3L1YfqeTR7M4oicgdh1Xde8e4BsJ6KFhtL+EeNnCHRHZ08DSLhYK\nmadpl3d6Ss0YsPJYfXqrqGcZka7gysyBiAEzOvgQbnkUeSbJ8zOy7JOIiDljbG0v3quUr2fnPhFf\nuUc5GpTaMWfWiBuIeO7IJUBEuXnZu7v7JRJu8yhPOrg2A7cCTazosFq5PZACUBNTDWb7v0q8T2gN\nkJqQFQ8koOk9p8o7vIVXiJdxHG/0e7Pkbj+TqEMuEZFzNKqXULZf80bTCvyoRliRg5IZQKpxdxtv\npnzF/v28eMttIvVbQ+XzR+2Rn3UhYTx77xgsC1YfyLvN6IZZIdwIVxN5gaP8zA6MIcuPCEBG4bzN\ni4l0QZUHKgqV452/L/ytImUUrgG+jTe1g8W7kDcNJBOP/JrXmK+yxZrXGHMfW15jRLF4jRHF4jVG\nFIvXGFEsXmNEsXiNEcXiNUYUi9cYUSxeY0SxeI0RxeI1RhSL1xhRLF5jRLF4jRHF4jVGFIvXGFEs\nXmNEsXiNEcXiNUYUi9cYUSxeY0SxeI0RxeI1RpT/Afe744xFBWU7AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "from torchvision.utils import make_grid\n",
    "\n",
    "# check for CD-1 generated data, they are similar to original data\n",
    "data, label = next(mnist01_loader(False, use_cuda, batch_size=32)())\n",
    "generated = rbm.contrastive_divergence(data, k=1)\n",
    "\n",
    "gs = plt.GridSpec(2,1)\n",
    "for irow, img in [(0, data), (1, generated)]:\n",
    "    img_grid = make_grid(img.view(32, 1, 28, 28).data)\n",
    "    npimg = np.transpose(img_grid.cpu().numpy(), (1, 2, 0))\n",
    "\n",
    "    plt.subplot(gs[irow,0])\n",
    "    plt.imshow(npimg)\n",
    "    plt.axis('off')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "fGVr6K_Y3ixs"
   },
   "source": [
    "## Quiz\n",
    "* Try to explain why the loss goes up.\n",
    "* How much ($t_{\\rm CPU}/t_{\\rm GPU}$) a GPU can accelerate in this case?\n",
    "* Try CD-10 training, and CD-10 reconstruction, tell the difference."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "daD390CW2NKH"
   },
   "source": [
    "## Task\n",
    "To see how RBM can be useful, try to solve the following problem.\n",
    "\n",
    "Top half of an image is lost, the only knowlege we know is it is a hand written number, try to recover this image.\n",
    "\n",
    "This image is:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 266
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1106,
     "status": "ok",
     "timestamp": 1525697596519,
     "user": {
      "displayName": "刘金国",
      "photoUrl": "//lh3.googleusercontent.com/-lDAT81T3HSE/AAAAAAAAAAI/AAAAAAAAAgw/eH3JEob7M1Y/s50-c-k-no/photo.jpg",
      "userId": "116824001998056121289"
     },
     "user_tz": -480
    },
    "id": "JGHW_08wc7Lb",
    "outputId": "b8655515-5b80-496c-e4d8-0f8cfb85f186"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAADD5JREFUeJzt3XuU13Mex/HPdJlqmrbpMmiaNFFs\nUumUXCNtLkW1dpE0OO5JmHNIWxsNx2LCtHZZ6ViLyNLiEN1mrW4iprRWlyFdZirddDWXpmZm/7D8\n93lNh+1XeT0f/758+v0a8+r7x/v7+XySampqAgA/dQ71FwBwaFB+wBTlB0xRfsAU5QdMUX7AFOUH\nTFF+wBTlB0zVS+SHTV3VXb5OOP7BoXJ9442V0axP/gdy7XubT5R5g9FNZL5+TPyr921bJNeu2pMu\n82Ur28j8X/3yZd7vhZHR7LpL/ynXjmrxpcxnljWQ+ZPr+8i85K120Szvtr/KtXcUDpb50I6FMm/f\ncHM0y108QK6965QCmafVLZP5A5OHyDz3mpej2ZSvT5dry/fXl3nBeROS5H/wPzz5AVOUHzBF+QFT\nlB8wRfkBU5QfMEX5AVMJnfPf8+5VMm/0690y79J6bTTrm7pMrp3xYG+Zr7tGn2iUe9Lr0ezRFRfI\ntRe1XSHzot1ZMu9TkCPz+mLse3Zj/Q5Cu7eHybzjY1tlnlStf257xlVEswnZeo4ff0PgOz1eWi3z\nu1+8PprV76J/16Zv7SzzqmuTZT51zuMyz8m+NZp9c3IjubY6uZYx/nk6/h5PfsAU5QdMUX7AFOUH\nTFF+wBTlB0xRfsBUQuf8Jz70lcwHzl0u87fPOiGa5ZadKde+tUrPXbPb633px/TfFc3qzEuTay+/\n42OZjxmizyLo9clNMv+8/7PR7MLMU+XaOvn63/9N+XqeXTO7hcyPzyyJZnmv/EOuXVaZIfPOydtk\nXtE2fv7DcU/rcwpSH9gr83U9s2SeWUuzyjIaRrP77pos1046tbv+w/+o4+/x5AdMUX7AFOUHTFF+\nwBTlB0xRfsBUQkd9oWmqjAc30cdIv9khPo7bdq8ezWRfeovMQ/UXMp6zp2M0233SPrl25O23yfy5\np/TR3AU9npH5BZePiGZl0/XW1dvbzJT5U+/0k3m95jIOv8uaHs2uWnyDXDvg+M9lPm9XfPQbQghN\nlsXHlJ3H6/Hrypv1Ue+b9Cnz4YrMM2S+cWJ1NMu7L1uu3TJOf/aB4skPmKL8gCnKD5ii/IApyg+Y\novyAKcoPmEronL/Os/FjnEMI4Tc33iHzxuM3RLMmeZly7U1TXpV5/lh9rPjCsVXRLC1LX5l8zaNv\nyPyKh+JXbIcQwvYe+2XecXN8a+v2Ba3k2llP6OvB2x8f38ocQgibztTbmaft7BbNzshcK9cu26W/\n+2XHLJb5mkXxWf3cip5y7ehX4ldohxDCpY23y3zguN4yb/dGfM5fp1L//06q1tusDxRPfsAU5QdM\nUX7AFOUHTFF+wBTlB0xRfsBUUk2NvmL5/6nDw/nyw/an6O9SrzR+NXFta7Pe1Xvu947cIfMNq1tG\nszaz5dKQOn+VzC+er88xKCo7Rual++PHUM+bo6+abqo/Osy5f4LMd1brmfSiivjx25NKzpFrd0zW\n7yDcOfo1mf9t2KBolpobf2ckhBA2lTaRefOr9Zw/t3CWzId8cHM0W9L7L3ptH73ff+bKR2q5w/s7\nPPkBU5QfMEX5AVOUHzBF+QFTlB8wRfkBUwmd8/f51cPyw6rGfCPXd20en83O33icXJueq/fcJ31R\nLPPiEfF5+Su36HP3c27U5/bv7KD3Z1c11GPbVgv2RLO6JVvk2uXj2so8c7b+7Mbry2TeddJ/olm7\nBlvl2rx5F8t87Llvy7x1/fi7G1O36avLP5zWReaFt+p7sE+fkCPzzEvWRrP72k6Ta7skx8+WCCGE\n1Ix1zPkBxFF+wBTlB0xRfsAU5QdMUX7AVEKP7v62tR5pbV6XLvOjU+IjrfJF8S23IYSwL2+9zOve\nq0eFWVNKotmQHvqq6YvyPpX58nMayTwpJUXnDeNbervO+Fqu/Wqa/ntv7KVHwZnv6+9eUBI/Pjvt\nT93l2vaj9XdfsKuDzJf8PT6e7TpYX//d7Mv40dohhJC9Wo8h0/vp37ecNgXR7OqP9O9Ts9n6Z174\nnIx/wJMfMEX5AVOUHzBF+QFTlB8wRfkBU5QfMJXQOX9Vst5p2KRlqczbNIpv0VycqufRA1p9JvPZ\nO/UWz+1ntY5m5V/rzx7UbYnMFw4aJvMLRs2X+Yz8+BHYs57Wx1+/PPoJmY/4vb42fe4zk2ReVl0Z\nzc499k65dttqfWT5+vX677a3fXzr6/zP4u8fhBBCalv9XNyx5WiZp01Nlfnja66MZlXD68q1+1IP\naMdurXjyA6YoP2CK8gOmKD9givIDpig/YIryA6YOq6O7K0btlOvrPhnfs5+6VF+5XLNPX9GdlKL3\nSJc+E5+tNrqnlv32pRUyT35Wv99QkaPPKih7JH58dmmlPrK83tQWMm/+un4/4oWV+n7yG1f/Npod\nl7pNrh3ecq7M+792t8wbtd8VzUqLfyHXpi/Ws/TKWmbtna/W5wVsHRZ/byR9ov5d7pSqzzkY3Wk6\nR3cDiKP8gCnKD5ii/IApyg+YovyAKcoPmErofv56C5fJfMdcfY57ylHx1wSKx2XItW/2fVLmj2zs\nJ/PlK+Ln2/eZuFKuLdp5lMz1FD+Ed9+ZLPP+KwdGs01L9J749h9ulvlrRe/J/LRFN8u8/Nv4nQLr\nmjbTa6v0Owpqjh9CCC0ax99/qNqWJtfeM/Ylmef9YajMS3JPkHmzP6+LZismdZJr55+t/+zRevkP\nePIDpig/YIryA6YoP2CK8gOmKD9givIDphI65/8i/xSZ1zSMn/EeQgjp/45vU76t11ty7ZWTc2Re\nr1RvgW6UHM+KZujB6tZT9L+xvQcskPkD2+L3zIcQwtDWH0WzKW/oWfqOHvodhM6zRsg8ZZX4wYQQ\nGotjFNIL5dKwfm26/uxzmsp8Xe/4OQstN+hzLPql6LMG8sv1+r1pulp7h8bff3h5wWNy7eClN8j8\nQPHkB0xRfsAU5QdMUX7AFOUHTFF+wFRCR30NtuirhzMW7Jd5yU3xUWDhnnZybYueeutq02y9PfSb\n/vFtlJXXb5drq3bq65o/veRYmZ88TR/lvLI8vp35q8v0Z1/c9xOZbyzX47TuPYtlnpkc/9mMrxos\n12Z8tEXmzZfqI9NbXLc7mhUXZcm1H+9tKPN65dUyLx6k8+fHvxrNhl97u1ybsVaPIcMAHX+PJz9g\nivIDpig/YIryA6YoP2CK8gOmKD9gKqFXdG/ZkCE/rO+jI+X6Xb+MvwfQq5s+Pnv+cn3c8QlZm2Re\n97L4Ndo3LFos104sOVfm+6v1v8EVz7WS+abz4z+X4T3fl2ufLzpd5uO7vi7zUZ/Fr+AOIYSxnaZH\nsxdP6yLX5hQulPn9Y/TW1qafx98xKB6oD0xP/1RvL+/4oL6Ce00vGYc1Y7pFs9Zz9Wdf9MQcmY86\naSZXdAOIo/yAKcoPmKL8gCnKD5ii/IApyg+YSuic//w6lyfuw3DYm7Vx6U9af2GGPgreVUH1VOb8\nAOIoP2CK8gOmKD9givIDpig/YIryA6YSem4//PzUWT4OHp78gCnKD5ii/IApyg+YovyAKcoPmKL8\ngCnKD5ii/IApyg+YovyAKcoPmKL8gCnKD5hiSy8OWxzNfXDx5AdMUX7AFOUHTFF+wBTlB0xRfsAU\n5QdMMeeHdDCP3maOf2jx5AdMUX7AFOUHTFF+wBTlB0xRfsAU5QdMMeeHVNssniu4j1w8+QFTlB8w\nRfkBU5QfMEX5AVOUHzBF+QFTzPnNHew5PXv2D188+QFTlB8wRfkBU5QfMEX5AVOUHzDFqA8/CaO8\nIxdPfsAU5QdMUX7AFOUHTFF+wBTlB0xRfsAUc/6fOY7WRgxPfsAU5QdMUX7AFOUHTFF+wBTlB0xR\nfsAUc35I7Nf/+eLJD5ii/IApyg+YovyAKcoPmKL8gCnKD5hizv8zwJ59/Bg8+QFTlB8wRfkBU5Qf\nMEX5AVOUHzBF+QFTzPmPAAdzjs9+fV88+QFTlB8wRfkBU5QfMEX5AVOUHzBF+QFTlB8wRfkBU5Qf\nMEX5AVOUHzBF+QFTlB8wxZbeIwDbbnEw8OQHTFF+wBTlB0xRfsAU5QdMUX7AFOUHTCXV1NQc6u8A\n4BDgyQ+YovyAKcoPmKL8gCnKD5ii/IApyg+YovyAKcoPmKL8gCnKD5ii/IApyg+YovyAKcoPmKL8\ngCnKD5ii/IApyg+YovyAKcoPmKL8gCnKD5j6L2TeiCUL8JlxAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# presented data, npdata[:14] is replace by noises, with rest the correct values.\n",
    "npdata = np.array([0.87207395, 0.30172917, 0.20748128, 0.8095541, 0.6876969, 0.1532767, 0.62968737, 0.7779726, 0.7118719, 0.53654927, 0.94739664, 0.4471923, 0.8174037, 0.99884754, 0.88779926, 0.71121424, 0.3754819, 0.8892206, 0.6074483, 0.80546534, 0.86275977, 0.59773445, 0.8402653, 0.86507905, 0.42217958, 0.87019306, 0.7207597, 0.70173293, 0.5013676, 0.632447, 0.7610868, 0.9116219, 0.27281326, 0.5140466, 0.85905105, 0.6330603, 0.7861834, 0.48230517, 0.5583505, 0.46642596, 0.9083495, 0.5351944, 0.1025035, 0.098153934, 0.5750544, 0.16645348, 0.16748504, 0.88547695, 0.43294504, 0.6214652, 0.71871835, 0.091364935, 0.93197143, 0.18161148, 0.34041223, 0.36640248, 0.05945792, 0.84343576, 0.05011359, 0.020642783, 0.96630734, 0.02089095, 0.8930233, 0.3763136, 0.9165772, 0.97183466, 0.63145226, 0.9990819, 0.009387237, 0.50575066, 0.64946336, 0.35694385, 0.69572824, 0.9233393, 0.86297685, 0.95064604, 0.5617286, 0.20560175, 0.7403321, 0.27093577, 0.97413605, 0.26959816, 0.87772864, 0.044952888, 0.102199115, 0.9718467, 0.16127962, 0.655842, 0.15577097, 0.019478982, 0.79594487, 0.5539078, 0.23640345, 0.9437329, 0.9587844, 0.17794718, 0.6766039, 0.8290011, 0.4769842, 0.4305203, 0.8977303, 0.6218739, 0.7924648, 0.76474386, 0.3790167, 0.77901715, 0.14843856, 0.49938354, 0.0016661843, 0.53694785, 0.28133965, 0.50130546, 0.7144231, 0.21561551, 0.71205837, 0.22371855, 0.87220854, 0.26955116, 0.23340793, 0.35401136, 0.8505834, 0.3398249, 0.5540171, 0.10345899, 0.42512947, 0.038393788, 0.14067493, 0.35083002, 0.78112274, 0.7566516, 0.7144876, 0.85403043, 0.17873417, 0.35017425, 0.8706891, 0.9905999, 0.03664582, 0.21388373, 0.19397263, 0.49474058, 0.45710087, 0.5013515, 0.45443046, 0.49812528, 0.20585743, 0.6777932, 0.52848434, 0.24695043, 0.05329345, 0.8382803, 0.7743761, 0.75724375, 0.47649342, 0.3232994, 0.97870475, 0.95442826, 0.8724979, 0.7017745, 0.31166396, 0.86247295, 0.19751902, 0.7225639, 0.66581416, 0.053260047, 0.91416854, 0.5880709, 0.97473216, 0.11180539, 0.19542904, 0.6878177, 0.77867174, 0.6552874, 0.15082805, 0.036130577, 0.40634972, 0.92630273, 0.20962614, 0.34388322, 0.37616885, 0.15954784, 0.06997797, 0.53446907, 0.2530169, 0.15803383, 0.79036176, 0.84305835, 0.54108804, 0.8501002, 0.8879858, 0.75407153, 0.42112336, 0.9809118, 0.38506842, 0.70881325, 0.9115492, 0.98289156, 0.29546624, 0.5558135, 0.73718935, 0.4230012, 0.6937958, 0.16916722, 0.08621408, 0.6282674, 0.78306913, 0.032396235, 0.05875099, 0.08453581, 0.7941298, 0.44791576, 0.49781203, 0.24143413, 0.6512627, 0.20860681, 0.697907, 0.7528845, 0.372322, 0.7488793, 0.27171475, 0.72370976, 0.69339174, 0.8918535, 0.5999, 0.30938724, 0.3181647, 0.64025414, 0.78654414, 0.47598284, 0.8310249, 0.11720193, 0.2878025, 0.52041465, 0.9025964, 0.28781494, 0.8044487, 0.3056101, 0.33250284, 0.92022175, 0.286686, 0.9976094, 0.9832922, 0.31882682, 0.55324644, 0.43472946, 0.5232928, 0.54625267, 0.3906454, 0.62188476, 0.27731606, 0.16046757, 0.5718505, 0.35741845, 0.19788711, 0.6796674, 0.23651811, 0.08174702, 0.06336878, 0.09723609, 0.6873774, 0.14071466, 0.10673229, 0.72016853, 0.14792068, 0.6807766, 0.65582806, 0.30159968, 0.033888057, 0.9616834, 0.71672386, 0.7785365, 0.93777555, 0.4879574, 0.6355177, 0.5111196, 0.3009759, 0.3500596, 0.74764377, 0.13936538, 0.7483063, 0.81750077, 0.022937937, 0.4070562, 0.31818643, 0.5540159, 0.92213124, 0.73755354, 0.5571023, 0.75489575, 0.67721623, 0.72045094, 0.9948514, 0.70227563, 0.4672006, 0.106317885, 0.9804078, 0.6265774, 0.5751128, 0.6412867, 0.7626526, 0.63349885, 0.6859269, 0.42727026, 0.87387127, 0.30769062, 0.21811083, 0.7262418, 0.3143378, 0.5617967, 0.52690506, 0.5614177, 0.2285414, 0.85394025, 0.4490645, 0.3991946, 0.4290848, 0.260644, 0.45061842, 0.79720527, 0.8023241, 0.8577737, 0.15940316, 0.28310093, 0.5588658, 0.40793347, 0.3871484, 0.19407174, 0.090147026, 0.3032796, 0.56522983, 0.31781578, 0.9819852, 0.27190876, 0.27358428, 0.08329663, 0.9401394, 0.64813715, 0.43394515, 0.20502856, 0.6062421, 0.77016854, 0.6724567, 0.05586233, 0.30247933, 0.6605889, 0.59062487, 0.07923536, 0.79024094, 0.7947558, 0.47801945, 0.7789812, 0.7338063, 0.81782705, 0.86098266, 0.37724164, 0.025790595, 0.13603827, 0.70920366, 0.39994055, 0.9000277, 0.24124062, 0.556982, 0.9389691, 0.19500403, 0.0904483, 0.9657381, 0.36499324, 0.53046125, 0.7467004, 0.558991, 0.70214766, 0.047792256, 0.8731797, 0.6984725, 0.6879274, 0.28380287, 0.5657824, 0.82289946, 0.628091, 0.8842273, 0.6312523, 0.86718184, 0.046136532, 0.89170617, 0.27153188, 0.12342716, 0.4058729, 0.21863484, 0.7674653, 0.026953213, 0.44575384, 0.20404944, 0.8207133, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], dtype='float32').reshape([28, 28])\n",
    "plt.imshow(npdata)\n",
    "plt.axis('off')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "AbrkPDTZCNMK"
   },
   "source": [
    "Here, I will give you some **hint**.\n",
    "\n",
    "* Train a better RBM to complete this task.\n",
    "\n",
    "* Make this image a initial input, and perform Gibbs sampling sounds a good idea. Try to write a modified version of **contrastive_divergence** method.\n",
    "* data transform related API (pytorch API page: http://pytorch.org/docs/0.3.0/): \n",
    "\n",
    "```python\n",
    "    torch.from_numpy\n",
    "    torch.Tensor.numpy\n",
    "    torch.Tensor.cuda\n",
    "    torch.Tensor.cpu\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "y1-nQVgdHxX8"
   },
   "source": [
    "## Solution\n",
    "Answers to Quizzes:\n",
    "1. The \"loss\" in the program is $\\tilde{\\mathcal{L}}=\\langle E_\\theta(x)\\rangle_{x\\sim \\mathcal{D}}-\\langle E_\\theta(x)\\rangle_{x\\sim p_{\\theta}(x)}$, which is not the actual one. The back propagation of $\\tilde{\\mathcal{L}}$ do not take the dependacy of $p_\\theta(x)$ into consideration, so it is the derivative of the true loss $\\mathcal{L}$ but not $\\tilde{\\mathcal{L}}$.\n",
    "2. Approximately $3$, with larger $k$ (CD-$k$), this ratio can become larger.\n",
    "3. CD-10 generationn with CD-1 training is not likely to generate reasonable digits, however, CD-10 training is better.\n",
    "\n",
    "To complete the image restoration task, I used CD-10 in the training of RBM."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "L5vfreav2hx0",
    "outputId": "d4f4cbe1-a927-401a-df9c-a4af540fed53"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 0, Mean \"Loss\" = -0.8171, Elapse = 27.3622\n",
      "epoch 1, Mean \"Loss\" = -3.9773, Elapse = 26.8473\n",
      "epoch 2, Mean \"Loss\" = -2.5730, Elapse = 26.9036\n",
      "epoch 3, Mean \"Loss\" = -1.5249, Elapse = 31.0247\n",
      "epoch 4, Mean \"Loss\" = -0.7390, Elapse = 36.6229\n",
      "epoch 5, Mean \"Loss\" = -0.2368, Elapse = 38.6045\n",
      "epoch 6, Mean \"Loss\" = 0.2680, Elapse = 31.3658\n",
      "epoch 7, Mean \"Loss\" = 0.5554, Elapse = 29.7760\n",
      "epoch 8, Mean \"Loss\" = 0.8041, Elapse = 39.4627\n",
      "epoch 9, Mean \"Loss\" = 0.9416, Elapse = 32.7747\n",
      "epoch 10, Mean \"Loss\" = 1.0861, Elapse = 31.9140\n",
      "epoch 11, Mean \"Loss\" = 1.1597, Elapse = 33.7012\n",
      "epoch 12, Mean \"Loss\" = 1.4048, Elapse = 30.0620\n",
      "epoch 13, Mean \"Loss\" = 1.5296, Elapse = 29.7897\n",
      "epoch 14, Mean \"Loss\" = 1.6287, Elapse = 28.4921\n",
      "epoch 15, Mean \"Loss\" = 1.7750, Elapse = 28.4476\n",
      "epoch 16, Mean \"Loss\" = 1.7078, Elapse = 28.1143\n",
      "epoch 17, Mean \"Loss\" = 1.8561, Elapse = 28.5129\n",
      "epoch 18, Mean \"Loss\" = 1.8869, Elapse = 34.0963\n",
      "epoch 19, Mean \"Loss\" = 1.8651, Elapse = 27.9472\n"
     ]
    }
   ],
   "source": [
    "# define network and dataset, and transfer model data into GPU memory\n",
    "rbm = RBM(num_visible, num_hidden)\n",
    "if use_cuda: rbm = rbm.cuda()\n",
    "loader = mnist01_loader(True, use_cuda, batch_size=64)\n",
    "\n",
    "# with the stochastic gradient descent optimizer,\n",
    "# we optimize model parameters with learning rate 0.1.\n",
    "train_op = torch.optim.SGD(rbm.parameters(), 0.1)\n",
    "for epoch in range(20):\n",
    "    t0 = time.time()\n",
    "    loss_list = []\n",
    "    for data, label in loader():\n",
    "        # calculate the \"loss\", the last node in computation graph\n",
    "        v1 = rbm.contrastive_divergence(data, k=10)  #! the only line changed.\n",
    "        loss = rbm.energy(data) - rbm.energy(v1)\n",
    "        loss_list.append(loss.item())\n",
    "\n",
    "        # get gradients using back propagation.\n",
    "        # zero_grad are needed before backward, otherwise gradients are accumulated.\n",
    "        train_op.zero_grad()\n",
    "        loss.backward()\n",
    "        # update parameters using gradients using gradients.\n",
    "        train_op.step()\n",
    "    t1 = time.time()\n",
    "    print('epoch %d, Mean \"Loss\" = %.4f, Elapse = %.4f'%(epoch, np.mean(loss_list), t1-t0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 283
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1291,
     "status": "ok",
     "timestamp": 1525696386622,
     "user": {
      "displayName": "刘金国",
      "photoUrl": "//lh3.googleusercontent.com/-lDAT81T3HSE/AAAAAAAAAAI/AAAAAAAAAgw/eH3JEob7M1Y/s50-c-k-no/photo.jpg",
      "userId": "116824001998056121289"
     },
     "user_tz": -480
    },
    "id": "2wFcTG3fGYG-",
    "outputId": "825aef6d-d5a5-4a21-bc5d-95fcb48248be"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Try to recover it using above trained RBM.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAADElJREFUeJzt3Wlw1dUdxvETk0AIS9iDbAHSEEAR\nlFVksQiiiFK2WqhSkLEgrYqgoB2hMi5IcSo6RdQRdKoFZ1QUF8oWBawKKpsii4AwIMgiOzGJQNIX\nLu/OA6NyEZ7v5+3j8d6Q++T/4nfPOUklJSUBgJ/zzvQbAHBmUH7AFOUHTFF+wBTlB0xRfsAU5QdM\nUX7AFOUHTKUk8sWuajhafp2wePtOuX7jwxfH15YplmsrfqJ/1Flj/iHzHh8PjWZ179ffktzbIkPm\n+9ocl/mKqx+Tebtpd0azKmtOyLXl3lgl86Oza8l8x87KMk/dkxrNsh9cI9cOXbFS5pNv6y/znQO/\njWaX1N0u13auvF7mE97pIfOst/RnIrkg/nlNPRJ/3yGEUFg9TeZL3hidJP+D7/HkB0xRfsAU5QdM\nUX7AFOUHTFF+wBTlB0wldM7f6VU91316VXuZj201K5r1LLtVrr2xUR+Z37BuoMz/3vTNaDaxzQC5\nNqVAxqH+y3omPKXtJfr/nx/PytyivztRUNRM5ofn649I4+mfyXxvrybRrPvSbXJtfnFpmZc6qOfh\n1V+Oz8OXdf2NXLu9XkWZP9vtGZkPPRD/XkgIIWQ/Ev8ewew1C+XaTceKZB7C6JPk3+HJD5ii/IAp\nyg+YovyAKcoPmKL8gCnKD5hK6Jz/jR1NZd654ecy75b+RTQ7qLfzh9zyu2U+a31Lmd9fcE00qztf\nz9K39dN74o/1PSLzD65rKPPQLx7lVNgrly5pUkfmJa0PyXxdTq7My26Nby2fOjP+bxpCCHXnHpZ5\n8jE9579gTPzzVKF7uly7cbT+N6/ZWP/OarbWn4kvn60Rza7NaiPXpuVVkfnrdWX8I578gCnKD5ii\n/IApyg+YovyAKcoPmEroqK98Hz1u67B8g8z/dP1fotnX9xTKtZlj9d+5hqs/lPnBgZdGs11d9dHc\nR3P1SKrB5HIy39ldj3by68WP53605rtybY8rq8k89TY9Els3Us9Yy2+L55XeWivXFrxSSeYHX9Mj\n1Bsy5kezex/qLdeGYn3k+X1f6qO7Ux7QR5rXWrEpmq2fGj+iPoQQKv/nJLXtoOMf8OQHTFF+wBTl\nB0xRfsAU5QdMUX7AFOUHTCWVlOhjo39JXdvpu6z3XVBWrk///a5oNib7v3LtmgK9dfWpJZ1l3njS\nV9Fs64Dacm373+mrpkdnLpB578n6KOZa876OZgea6Xnz3hYyDiUnGSknF+rboI+Xif/Ks1/S3804\nWkdfRZ1SpD+7h+skR7PBf54j1y7o0kjmNy1+X+aTxuvj3AuqxZ+7bQfoz8s7C5vLfNPdI7miG0Ac\n5QdMUX7AFOUHTFF+wBTlB0xRfsBUQvfzH8rWe8P3XXZM5n/Nei+atSm9T67914V6Npp670n+DhbF\n9+QXZOo97Vlp+2U+vH4nmb+0ZZLMs+8sE82ePJQl1z7x/LUyr/3wBzI/cbnee36oXvya7R2X689D\n2j49xz9wvh5nX99zcTR7/rGr5drqmfrI8umd2sm87ZsfyzzvxdbRbNGWHLn2o4H/lHkII0+Sf4cn\nP2CK8gOmKD9givIDpig/YIryA6YoP2AqoXP+yiv0LP5Eqaoyr9ExPntdVqTPtk/K0nvuayzT57Rv\nGFUvmjWetF2uvaW33p+dsUZfc93rqbtkPmXIk9Fs8uv6fPnMTvEzEkIIYdzNy2VeNmmpzN/ObxzN\n5g/Ss/IXXn1K5tfdNUrmM9I6RrPjl+jfd/WlJ7nzvXQpGQ+tskTmiwrjc/7MGfocg/73DZT53M9k\n/COe/IApyg+YovyAKcoPmKL8gCnKD5hK6Khvz2V6lFdhm97S26xUfFS44VgFuXbdKH3dc7lqR2V+\nY4NV0Wx5q7py7YAcfSx4aNRAxlc885HMhz83LJq9MkRv/+w55zaZTyqjt75ueCdb5rUWxY/nTi38\nRq5dUVRR5gVV9LOrwd3LotnmifFRWwghPDD73zJ/cPs1Mu8xS2+rvXZQ/Er43pX0duAJvfrL/FTx\n5AdMUX7AFOUHTFF+wBTlB0xRfsAU5QdMJXTOn9Z3t8x3L64h80F120ezzTP00dwZn6bK/NCF+nrw\n9x6Jz4UPNYgfnR1CCKGfjqvM2SDzefNayrzqhvj201WFeitzuS36I7C2cqbMm3fR7z1/Wvzf5pvp\n8Su0QwhhwpbuMk/fo7flNl8ez5ParpBrR3xwq8yrjfhC5sOu1Neuz5jaLZp9vrCeXLt5sP7+w6ni\nyQ+YovyAKcoPmKL8gCnKD5ii/IApyg+YSiop0dcg/5JmbGwtX2zakJ5y/bar4zPj5wc8LtcO+/QG\nmVeYqs8DyLlvbTTbOL6JXHusnJ5nl96vzzEoSdF/o8ss/TyazVwzV679w1WDZb67vT4Hofuw/8m8\nV0b86O86KfrnHvtVF5m/O1tfD35R9/XRbNNz+rj0Gn/cKvPi4eVlvu6ODJlP6PhyNBu3Ul+bPqbZ\nfJnfnPuuvrv8ezz5AVOUHzBF+QFTlB8wRfkBU5QfMEX5AVMJnfN3yrtTvtj2vXqmXP799GhWWFm/\ndsWN+srlA73yZd6ydvwa7v1F8fcVQgjrNteUefoX+rrnaiv1PHz3TfGz8ZOT9c99YrWeR3+bUyDz\n4sP6nIR6r8d/5Yez9FkCxSl6XL30b4/J/OInbo9mtZbon2tHR31GQ9bk1Xr9sGYyr7j5eDS76eHX\n5NotRdVkfn/T15jzA4ij/IApyg+YovyAKcoPmKL8gCnKD5hK6Ln9+/P1PPy32Rtl/snci6JZh0H6\nTvO+lfUd9xM76vvW33+wQTTLfTQ+Zw8hhCa74t8RCCGEE7X13HZLL713fFLzV6LZlJyGcu22ce1k\nnvuQ/v7DeU8ckXnJxPhH7KtL68i15bfq76C0eDw+xw8hhLJ7xHcM6qXJtYP7z5P5oplNZd55wIcy\nzykTv8MiLUl/r+O929vIPLyt4x/w5AdMUX7AFOUHTFF+wBTlB0xRfsBUQkd9+Zv09tEdg/TbSe4W\nH91saKnHIyOG3SLzzCoHZT6qVfzK5W9e0Fty8wbrcdruVnqUd0+f+DHPIYRwx5sDo1nZkfrv+7Fc\nvbW16rRdMh9fc47Mh+SOiGaVWusr24801eO49Nn6uPUTffdFs/719Wh4YYcsmW8afb7M9+/Rx7Xn\n5cWvfK+1WI9PU9fqa9FPFU9+wBTlB0xRfsAU5QdMUX7AFOUHTFF+wFRC5/wNn94r8623XiDzC7vF\n55urL7pUrl1w4ySZr/22isyn72ofzT7J09c9d5qyUubJffRrv/hRN5nXqB/PWo3RW0szUvScf3y1\nz2Te/Qp9xfeWUfFseJ1Vcm1ei6oyL+wS3+IdQgjpg76OZs8Mu0quHbd0pszHzm4k80p6t3GotGtN\nNCtsnSPXls7WW6FPFU9+wBTlB0xRfsAU5QdMUX7AFOUHTFF+wFRCr+juel6/xL0YfvXm7dRz/pPp\nVrP5L/ROzi0Lil/iim4AcZQfMEX5AVOUHzBF+QFTlB8wRfkBUwndzw8/P3eWj9OHJz9givIDpig/\nYIryA6YoP2CK8gOmKD9gijk/frXYr3968eQHTFF+wBTlB0xRfsAU5QdMUX7AFKM+/Cxs2T178eQH\nTFF+wBTlB0xRfsAU5QdMUX7AFOUHTDHnxxnDlt0ziyc/YIryA6YoP2CK8gOmKD9givIDpig/YIo5\nP6Sfu1+fWf6vF09+wBTlB0xRfsAU5QdMUX7AFOUHTFF+wBRzfnOcu++LJz9givIDpig/YIryA6Yo\nP2CK8gOmGPWd4073KI8tu2cvnvyAKcoPmKL8gCnKD5ii/IApyg+YovyAKeb8kJjjn7t48gOmKD9g\nivIDpig/YIryA6YoP2CK8gOmKD9givIDpig/YIryA6YoP2CK8gOmKD9givIDptjPfw7gmm38FDz5\nAVOUHzBF+QFTlB8wRfkBU5QfMEX5AVOUHzBF+QFTlB8wRfkBU5QfMEX5AVOUHzDFlt6zwOncsssV\n3L548gOmKD9givIDpig/YIryA6YoP2CK8gOmmPOfBZjF43TgyQ+YovyAKcoPmKL8gCnKD5ii/IAp\nyg+YSiopKTnT7wHAGcCTHzBF+QFTlB8wRfkBU5QfMEX5AVOUHzBF+QFTlB8wRfkBU5QfMEX5AVOU\nHzBF+QFTlB8wRfkBU5QfMEX5AVOUHzBF+QFTlB8wRfkBU5QfMPV/yclxYUu8kYAAAAAASUVORK5C\nYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# this is how I generate this data, to prevent me from cheating\n",
    "data, label = next(mnist01_loader(False, use_cuda, batch_size=32)())\n",
    "npdata_raw = data.data[0].cpu().numpy().reshape([28, 28])\n",
    "print('Try to recover it using above trained RBM.')\n",
    "npdata_gen = npdata_raw.copy()\n",
    "npdata_gen[:14]=np.random.random([14,28])\n",
    "plt.imshow(npdata_gen)\n",
    "plt.axis('off')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "YbG3vkiY5HDr"
   },
   "outputs": [],
   "source": [
    "def masked_contrastive_divergence(rbm, v, k):\n",
    "    '''\n",
    "    the modified version of contrastive divergence.\n",
    "    \n",
    "    Args:\n",
    "        v (ndarray): visible input, with bottom half of figure unchanged.\n",
    "        k (in): CD-k, means k times v->h & h->v sweep in a single contrastive divergence run.\n",
    "\n",
    "    Returns:\n",
    "        ndarray: visible obtained through CD sampling.\n",
    "    '''\n",
    "    prob_h = rbm._v_to_h(v)\n",
    "    h = sample_from_prob(prob_h)\n",
    "    for _ in range(k):\n",
    "        prob_v = rbm._h_to_v(h)\n",
    "        v.data.view([28, 28])[:14] = sample_from_prob(prob_v).data.view([28, 28])[:14]\n",
    "        prob_h = rbm._v_to_h(v)\n",
    "        h = sample_from_prob(prob_h)\n",
    "    return v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "e6RpLVWu8s5S"
   },
   "outputs": [],
   "source": [
    "# we should pack numpy data into tensors on CPU/GPU before traning.\n",
    "data = torch.from_numpy(npdata.reshape([1,784]))\n",
    "if use_cuda: data = data.cuda()\n",
    "res = masked_contrastive_divergence(rbm, data, 50)\n",
    "# dowload data from GPU and turn it into numpy array.\n",
    "res = res.cpu().data.numpy().reshape([28, 28])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 549
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1287,
     "status": "ok",
     "timestamp": 1525696390631,
     "user": {
      "displayName": "刘金国",
      "photoUrl": "//lh3.googleusercontent.com/-lDAT81T3HSE/AAAAAAAAAAI/AAAAAAAAAgw/eH3JEob7M1Y/s50-c-k-no/photo.jpg",
      "userId": "116824001998056121289"
     },
     "user_tz": -480
    },
    "id": "86sHl4bj-XHS",
    "outputId": "118a6119-ff16-4a96-98f6-8eafa234fc61"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data we recovered.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAA8BJREFUeJzt3dFt20AUAMHIcBWpIk0EriBVugIj\nTaQKl2G6gpCC5dPxtDPfhkhYWLyPx6Mu27b9AHqeZt8AMIf4IUr8ECV+iBI/RIkfosQPUeKHKPFD\n1PM9L/b76Y/HCWGwvx+vl2v+zuSHKPFDlPghSvwQJX6IEj9EiR+ixA9R4oco8UOU+CFK/BAlfogS\nP0SJH6LED1HihyjxQ5T4IUr8ECV+iBI/RIkfosQPUeKHKPFDlPghSvwQJX6IEj9EiR+ixA9R4oco\n8UOU+CFK/BAlfogSP0SJH6LED1HihyjxQ5T4IUr8ECV+iBI/RIkfosQPUeKHKPFDlPghSvwQJX6I\nep59A6zt7f3f7Fv4kpefv2bfwnQmP0SJH6LED1HihyjxQ5T4IUr8EGXP/+CO9vBH++5V9/gcM/kh\nSvwQJX6IEj9EiR+ixA9R4ocoe/4TmLlLH33tvecIbn0G4cje54++9gpMfogSP0SJH6LED1Hihyjx\nQ5T4Icqe/xusfOZ95j575B6fYyY/RIkfosQPUeKHKPFDlPghSvwQZc+/gMLZcu7P5Ico8UOU+CFK\n/BAlfogSP0RZ9V3J8dH7G/k/tz41+SFL/BAlfogSP0SJH6LED1Hihyh7/hOwc2YGkx+ixA9R4oco\n8UOU+CFK/BAlfoiy57+SXfz3u/W8vu/kNiY/RIkfosQPUeKHKPFDlPghSvwQZc/PUH7v4LxMfogS\nP0SJH6LED1HihyjxQ5T4IUr8ECV+iBI/RIkfosQPUeKHKPFDlCO9nJZXc49l8kOU+CFK/BAlfogS\nP0SJH6LED1H2/Owa+epte/y5TH6IEj9EiR+ixA9R4oco8UOU+CHKnp9dR7t4P8G9LpMfosQPUeKH\nKPFDlPghSvwQJX6IsuePG72nd2b/vEx+iBI/RIkfosQPUeKHKPFDlFUfN7HKW5fJD1HihyjxQ5T4\nIUr8ECV+iBI/RNnzPziv1uZ/TH6IEj9EiR+ixA9R4oco8UOU+CHKnp9dzus/LpMfosQPUeKHKPFD\nlPghSvwQJX6Isud/AM7s8xUmP0SJH6LED1HihyjxQ5T4IUr8EGXPv4CRe3zn9btMfogSP0SJH6LE\nD1HihyjxQ5T4IUr8ECV+iBI/RIkfosQPUeKHKPFDlCO9C3DslhFMfogSP0SJH6LED1HihyjxQ5T4\nIeqybdvsewAmMPkhSvwQJX6IEj9EiR+ixA9R4oco8UOU+CFK/BAlfogSP0SJH6LED1HihyjxQ5T4\nIUr8ECV+iBI/RIkfosQPUeKHqE+hCEgZS8KHngAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The original untouched data.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAA7JJREFUeJzt3dFNwmAYQFEhTOEULmGcwCmdwLiE\nUziGdQJpw08p5Z7zqlZCcvM9fG3/wzRNT0DPcesPAGxD/BAlfogSP0SJH6LED1HihyjxQ5T4Iep0\ny3/2enx3OyGs7Ov347Dk90x+iBI/RIkfosQPUeKHKPFDlPghSvwQJX6IEj9EiR+ixA9R4oco8UOU\n+CFK/BAlfogSP0SJH6LED1HihyjxQ5T4IUr8ECV+iBI/RIkfosQPUeKHKPFDlPghSvwQJX6IEj9E\niR+ixA9R4oco8UOU+CFK/BAlfogSP0SJH6LED1HihyjxQ5T4IUr8ECV+iBI/RIkfosQPUeKHqNPW\nH2AvPn++//3Z2/PLxX+7xNz14RImP0SJH6LED1HihyjxQ5T4IUr8EGXPv9C5XfvoHn/O2tcvcu+E\nyQ9Z4oco8UOU+CFK/BAlfogSP0TZ89/A6E7Znv/65r7Twn0AJj9EiR+ixA9R4oco8UOU+CFK/BBl\nz38Fa++E97xzHjnvYOTac/b8nV6LyQ9R4oco8UOU+CFK/BAlfogSP0TZ87Mq+/T7ZfJDlPghSvwQ\nJX6IEj9EiR+irPq4W6OvLLdmPM/khyjxQ5T4IUr8ECV+iBI/RIkfouz52Yyjx7dl8kOU+CFK/BAl\nfogSP0SJH6LED1HihyjxQ5T4IUr8ECV+iBI/RIkfosQPUZ7nZ7e8l3+MyQ9R4oco8UOU+CFK/BAl\nfogSP0TZ83O37PHXZfJDlPghSvwQJX6IEj9EiR+irPoYMnLMtlXetkx+iBI/RIkfosQPUeKHKPFD\nlPghSvwQJX6IEj9EiR+ixA9R4oco8UOU+CFK/BAlfogSP0SJH6LED1HihyjxQ5T4Icp7++NG3ru/\nhHfz3y+TH6LED1HihyjxQ5T4IUr8EGXVxxCrvP0y+SFK/BAlfogSP0SJH6LED1Hihyh7/gc3+siu\nPf7jMvkhSvwQJX6IEj9EiR+ixA9R4oco8UOU+CFK/BAlfogSP0SJH6LED1HihyjP8+/A2sdo02Ty\nQ5T4IUr8ECV+iBI/RIkfosQPUeKHKPFDlPghSvwQJX6IEj9EiR+iPNK7A3PHZI888usI7i6TH6LE\nD1HihyjxQ5T4IUr8ECV+iLLnfwB29VzC5Ico8UOU+CFK/BAlfogSP0SJH6IO0zRt/RmADZj8ECV+\niBI/RIkfosQPUeKHKPFDlPghSvwQJX6IEj9EiR+ixA9R4oco8UOU+CFK/BAlfogSP0SJH6LED1Hi\nhyjxQ9QfCWs8ee7FldUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print('Data we recovered.')\n",
    "plt.imshow(res)\n",
    "plt.axis('off')\n",
    "plt.show()\n",
    "\n",
    "print('The original untouched data.')\n",
    "plt.imshow(npdata_raw)\n",
    "plt.axis('off')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "PYqgj6DE-8MM"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "default_view": {},
   "name": "rbm_generation_with_solution.ipynb",
   "provenance": [],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
